{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Autoencoder\n",
    "- Convolutional autoencoders using convolutional & pooling layers can be generated as well\n",
    "- Fundamentals are identical; just replace dense layers with convolutional layers!\n",
    "\n",
    "<br>\n",
    "\n",
    "<img src=\"https://camo.githubusercontent.com/c2b4e51b1ebacac0d5fae4796bff2572797cc385/687474703a2f2f6d692e656e672e63616d2e61632e756b2f70726f6a656374732f7365676e65742f696d616765732f7365676e65742e706e67\" style=\"width: 800px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn import datasets\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dataset\n",
    "- Digits dataset in sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_data = data.images\n",
    "y_data = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fit in data instances into interval [0,1]\n",
    "X_data = X_data / 16."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_data = X_data.reshape(X_data.shape[0], X_data.shape[1], X_data.shape[2], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size = 0.3, random_state = 777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1257, 8, 8, 1)\n",
      "(540, 8, 8, 1)\n",
      "(1257,)\n",
      "(540,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating convolutional autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"input_4:0\", shape=(?, 8, 8, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "inputs = Input(shape = (8,8,1))\n",
    "print(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 4, 4, 5)\n"
     ]
    }
   ],
   "source": [
    "code = Conv2D(5, (3, 3), activation='relu', padding='same')(inputs)\n",
    "code = MaxPooling2D((2,2), padding = 'same')(code)\n",
    "print(code.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 8, 8, 5)\n"
     ]
    }
   ],
   "source": [
    "# in order to \"decode\" the code representations, upsampling methods can be used\n",
    "decoded = Conv2D(5, (3,3), activation = 'relu', padding = 'same')(code)\n",
    "decoded = UpSampling2D((2,2))(decoded)    \n",
    "print(decoded.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 8, 8, 1)\n"
     ]
    }
   ],
   "source": [
    "# note that output shape should be identical to the input shape\n",
    "outputs = Conv2D(1, (3, 3), activation = 'sigmoid', padding = 'same')(decoded)    \n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "auto_encoder = Model(inputs = inputs, outputs = outputs)\n",
    "auto_encoder.compile(optimizer = 'adam', loss = 'binary_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1d382691f60>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auto_encoder.fit(X_train, X_train, epochs = 1000, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoded = auto_encoder.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d3f94578d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAADsCAYAAAB37KKJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADrFJREFUeJzt3VuoXeW5BuBvmsRDKE1Qg9ZDtcVqK40HtIIFMfcWjKCV\nGqGJUixVqSmoqHjoAVQQVq5a6E1SQXvhgaAg4oUa0AttQyNVMZbWhKbgIWoi2hiNjn3RbuyGvcl6\n114zK2t+z3P9zvGPOf415nwzAvMbDcNQAABdHTLXJwAAMJeUIQCgNWUIAGhNGQIAWlOGAIDWlCEA\noDVlCABoTRkCAFpThgCA1hYm4dFoFP1c9YIFC6KTOe6446L8EUccEeUXLozeblVV7d27N8r/9a9/\njddIDMMwmo3jpHs5bieffHKUX7x4cbzG1q1bo/xnn30Wr5GYrb2smv/7edRRR8VrbNu2Lcq/++67\n8RqJ+XJvptf6xBNPHNOZfCG9N/fs2TOmM/mX+bKXqfQ7efny5fEa6X25a9eueI3QzmEYlu0vNErG\ncaQbu3Tp0iRed911V5Q/66yzonx6PlX5xq5cuTJeIzGpN+mGDRuifLr3VVUrVqyI8uO+SSe5DKX7\n+cMf/jBeY82aNVE+PafUfLk3V69eHeXXrVs3nhP5D+m9uWXLlvGcyL/Nl71Mpd+B6fdfVf73tXHj\nxniN0OZhGM7dX8h/kwEArSlDAEBryhAA0JoyBAC0pgwBAK0pQwBAa8oQANCaMgQAtKYMAQCtKUMA\nQGv5sK5AOp8o/RnvdPTFAZiBMrHSn3FPRx+M+/hV9v//I/3J/HRcyiWXXBLlqw7Iz/jPC+nn4Pr1\n66P8Sy+9FOXTz/2q/D2MexzHpErvmSVLlox9jYOFJ0MAQGvKEADQmjIEALSmDAEArSlDAEBryhAA\n0JoyBAC0pgwBAK0pQwBAa8oQANCaMgQAtHZQzSZL582k849mMptqJjOwJlE6O+ziiy+O8ps2bYry\n69ati/L8T+m9k+7n1772tSi/bdu2KM8X0r1MZ42lxzc37MC54YYbovyFF144pjP5wnz9e/FkCABo\nTRkCAFpThgCA1pQhAKA1ZQgAaE0ZAgBaU4YAgNaUIQCgNWUIAGhNGQIAWlOGAIDWxjqbLJXOTUln\nn81kNlk6k2tSZ2al1zo1k71h5lauXBnlf/e730V5s8YOnLvuumus+XTW1Jlnnhnlq6pWr14dv2YS\npffl1NRUlP/5z38e5e+8884oX5W/h9S4Zpl5MgQAtKYMAQCtKUMAQGvKEADQmjIEALSmDAEArSlD\nAEBryhAA0JoyBAC0pgwBAK0pQwBAa6NhGKYfHo2mH658rtcNN9wQ5dMZOzOZr7Vhw4Yov2LFiniN\nxDAMo9k4TrqX6ft65plnovzu3buj/ExmwD377LNjzadmay+r8v1MZ0GtX78+ym/fvj3Kz2TeUHr/\nj2um0X+bq3tz3DZu3Bjl08/9qvF/bqbmai/Ta33xxRdH+U2bNkX5dF7oTKTnNIO/lc3DMJy7v5An\nQwBAa8oQANCaMgQAtKYMAQCtKUMAQGvKEADQmjIEALSmDAEArSlDAEBryhAA0JoyBAC0pgwBAK0t\nHOfBd+3aFeXTwYsHwkyGu5JbsmRJlF+5cmW8xp133hnl16xZE+XTob5zKR1Cmw5eTe+bdEhzVX69\nzzrrrHiNSZReh3QY6CWXXBLlq/LBy5Pw3TIXDsQ9sHbt2iifDqcdF0+GAIDWlCEAoDVlCABoTRkC\nAFpThgCA1pQhAKA1ZQgAaE0ZAgBaU4YAgNaUIQCgNWUIAGhtNAzD9MOj0fTDM7B69eoon85Xmsl8\nmnSNcc+nGoZhNBvHGfdeprOmpqamonw6/6Yq//taunRplE/ncc3WXlaNfz/Tv+sDMTtq27ZtUT6d\ny5Qef77cm+m1nsncuHFbsWJFlN+yZUuUn6u9TD9zxv03/cYbb0T5qqrRaNY+1mbL5mEYzt1fyJMh\nAKA1ZQgAaE0ZAgBaU4YAgNaUIQCgNWUIAGhNGQIAWlOGAIDWlCEAoDVlCABoTRkCAFpbONcn8J9W\nrlwZ5devXx/lX3rppShfdXDO5ZkP1q1bN9bjz2SWVSqdZTbJ0muR7s/7778f5auqdu/eHeXTeWmT\nKp3TtWTJkii/adOmKF+Vf/ZP6l6m7yudnZnOT9y+fXuUn888GQIAWlOGAIDWlCEAoDVlCABoTRkC\nAFpThgCA1pQhAKA1ZQgAaE0ZAgBaU4YAgNaUIQCgtdEwDNMPj0bvVFWfYSUHn5OGYVg2Gweyl3Nu\n1vayyn4eBNybk8NeTpZp7WdUhgAAJo3/JgMAWlOGAIDWlCEAoDVlCABoTRkCAFpThgCA1pQhAKA1\nZQgAaE0ZAgBaU4YAgNaUIQCgNWUIAGhNGQIAWlOGAIDWlCEAoDVlCABoTRkCAFpThgCA1pQhAKA1\nZQgAaG1hEh6NRkOYj07m6KOPHms+PZ+qqrfeeivKv/vuu/EaiWEY8jfxv0j3MnXIIVnPPuaYY6L8\nUUcdFeWrql599dUo//nnn8drJGZrL6sOvv381re+FeUPP/zwKF9VtXXr1ij/4Ycfxmsk5su9+ZWv\nfCXKH3vssWM6ky+89tprUX7Pnj1jOpN/mS97mVq4MPrKr9NPPz1eI70v9+7dG68R2jkMw7L9hbIr\nE0ov/Pe///0of/XVV0f5BQsWRPmqqqmpqSh///33R/lxf+HOlrRIpl9u11xzTZRftWpVlK+qOuec\nc6L8Bx98EK8xqRYvXhzlH3744Sj/zW9+M8pXVV1wwQVR/rnnnovXmA/SovqjH/0oyt98881Rfib/\n6Pzud78b5bds2RKvQdXSpUuj/FNPPRWvceGFF0b5tDzNwPbphPw3GQDQmjIEALSmDAEArSlDAEBr\nyhAA0JoyBAC0pgwBAK0pQwBAa8oQANDaWH+Bevny5VH+qquuivK//vWvo/yf/vSnKF9V9frrr0f5\n+fKL0qn0V25vueWWKH/llVeO9fhVVf/85z/j10yq9NfYn3766Sh/xBFHRPm77747yldVbd68OX7N\nJPrpT38a5e+4444on35uzuTXxK+99toon/6K9qRatGhRlH/xxRejfDryqqpq+/Zp/eDzQceTIQCg\nNWUIAGhNGQIAWlOGAIDWlCEAoDVlCABoTRkCAFpThgCA1pQhAKA1ZQgAaE0ZAgBaG+tssmXLlkX5\nY489NsovXrw4yg/DEOWrqj799NP4NZPosMMOi/I//vGPo/x7770X5R9//PEoX5XPjUvnsc2nuXTf\n+MY3ovwJJ5wQ5c8///wov2PHjihfNbP7eRKtWLEiyv/+97+P8mvXro3yW7dujfJV+f0/qQ499NAo\nf+utt0b5448/Psrv27cvylfl3/vpvT+u+96TIQCgNWUIAGhNGQIAWlOGAIDWlCEAoDVlCABoTRkC\nAFpThgCA1pQhAKA1ZQgAaE0ZAgBaG+tssnSGyHHHHRflb7/99ii/ZcuWKF9Vdc8990T5p59+OsrP\nl/lKp5xySpQ/+uijo3y6NzOZGZde6/myNzORzg4b97VI58DNxHyaHZe44ooronx6Hb73ve9F+S99\n6UtRvqpqamoqfs0kuuiii6L89ddfH+VfeeWVKJ9+J1dVbdy4McqvWrUqyqez76b72eXJEADQmjIE\nALSmDAEArSlDAEBryhAA0JoyBAC0pgwBAK0pQwBAa8oQANCaMgQAtKYMAQCtjXU22Ysvvhjl16xZ\nE+UfffTRKJ/Oy6qq+u1vfxvln3/++Sj/8ccfR/m5cvjhh4/1+Mcff3yU/81vfhOv8Y9//CPK/+pX\nv4ryn3zySZSfS2+++WaUT2cUPfnkk1F+586dUb6q6oknnojy9957b5SfL7PMPvrooyh/6KGHRvm7\n7747yqd7X1W1a9euKL9o0aIoP5NZhnPhF7/4RZQ/8sgjo/wf/vCHKH/SSSdF+aqqM844I8qn7/ny\nyy+P8tPlyRAA0JoyBAC0pgwBAK0pQwBAa8oQANCaMgQAtKYMAQCtKUMAQGvKEADQmjIEALSmDAEA\nrSlDAEBrYx3Umg7f27Bhw3hO5N/27t0bv+aYY46J8osXL47y82VQ644dO6J8OuQyHdT6ne98J8pX\nVV122WVRftmyZVH+Jz/5ybSzwzBEx55tf/7zn6P8vn37ovwtt9wS5f/4xz9G+aqqX/7yl1E+HVA6\nX+7N1Ne//vUof+KJJ0b5n/3sZ1G+qurKK6+M8qeeemqUv+mmm6L8XPn73/8e5b/97W9H+aVLl0b5\nBQsWRPmqqk2bNkX566+/PsqP67PTkyEAoDVlCABoTRkCAFpThgCA1pQhAKA1ZQgAaE0ZAgBaU4YA\ngNaUIQCgNWUIAGhNGQIAWhslcz5Go1E0FOSQQ7Ku9dWvfjXKp8e/7bbbonxV1csvvxzl161bF+XT\nOSvDMIyiF/wf0r1MZ9Q89NBDUf7000+P8m+//XaUr6o67bTTovyePXui/CmnnDLt7L59+2ZtL6vy\n/Vy0aFF0/HRu4I033hjlv/zlL0f5qqoXXnghyp999tlR/m9/+1uUn6t7M/WDH/wgyj/44INR/oMP\nPojyVVXvvPNOlD///PPHevy52sv0PjjyyCOj/LnnnhvlH3jggShfVbV8+fIo//rrr8drhDYPw7Df\nN+7JEADQmjIEALSmDAEArSlDAEBryhAA0JoyBAC0pgwBAK0pQwBAa8oQANCaMgQAtKYMAQCtLRzn\nwdN5Vo888kiUP/PMM6P8X/7ylyhfVXXddddF+XTW2Hzx2WefRflLL700yqdz49auXRvlq6p27twZ\n5aempqL8fNr7Tz/9NMpfffXVUf7aa6+N8vfdd1+Ur6r6+OOPo/xbb70VrzGJ3n///SifzhrbvXt3\nlK+qOu+886L8e++9F68xH6TXOs2/+eabUf6xxx6L8lVVO3bsiF9zMPBkCABoTRkCAFpThgCA1pQh\nAKA1ZQgAaE0ZAgBaU4YAgNaUIQCgNWUIAGhNGQIAWlOGAIDWRsk8pdFo9E5VbR/f6bAfJw3DsGw2\nDmQv59ys7WWV/TwIuDcnh72cLNPaz6gMAQBMGv9NBgC0pgwBAK0pQwBAa8oQANCaMgQAtKYMAQCt\nKUMAQGvKEADQmjIEALT2X2bZmdo9rztzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d3f36e8438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n = 5\n",
    "plt.figure(figsize=(10, 4))\n",
    "for i in range(n):\n",
    "    # display original\n",
    "    ax = plt.subplot(2, n, i+1)\n",
    "    plt.imshow(X_test[i].reshape(8, 8))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "\n",
    "    # display reconstruction\n",
    "    ax = plt.subplot(2, n, i + n+1)\n",
    "    plt.imshow(decoded[i].reshape(8, 8))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
